from master.合并表 import hb as HBB
import operator
import math


class MapReduce(HBB):
    def __init__(self, READPATH=r'C:\Users\Seven\Desktop\新建文件夹 (3)\新建 Microsoft Excel 工作表.xlsx', MAX_I=1000, MAX_U=1000,
                 MAX_DICT_LIMIT=1000000, LISTNUMBER=20, GYH=0.0001):
        self.READPATH = input('请输入文件路径：\n')
        # self.READPATH = READPATH  # 读取数据文件路径
        self.WPATH = self.READPATH.rsplit('\\', 1)[0] + '\\导出.csv'
        self.pathListwLOG = self.READPATH.rsplit('\\', 1)[0] + '\\结果日志.LOG'
        self.MAX_I = MAX_I  # 关联1最大值
        self.MAX_U = MAX_U  # 关联2最大值
        self.MAX_DICT_LIMIT = MAX_DICT_LIMIT  # 共现字典分片限制值
        self.LISTNUMBER = LISTNUMBER  # 输出最大值
        self.GYH = GYH  # 归一化边界处理
        self.DATA0 = {}  # 初始共现矩阵
        self.dataList = ['self.DATA0']  # 共现矩阵，维护列表
        self.data = []  # map1
        self.g1_g2_s = []  # reduce1
        self.Co1_Co2_score = []  # 共现矩阵 临时生成列表

        # 流程控制
        self.MapReduce1()  # 数据归一化
        self.MapReduec2()  # 数据共现矩阵
        del self.g1_g2_s  # 释放部分无效数据
        self.MapReduce3()  # 相似度计算

    # 生成共现矩阵
    def CoCos_dict(self, Co1_Co2_s):
        if Co1_Co2_s == []:
            return self.dataList
        flag = None
        flag1 = False
        s_sum = 0.0
        for g_g_s in Co1_Co2_s:
            g1, g2, s = g_g_s
            if g1 == g2:
                continue
            if flag is None:
                flag = g1 + '\t' + g2
            for dict_i in self.dataList:
                dict_i = eval(dict_i)
                if (g1, g2) in dict_i.keys():
                    dict_i[(g1, g2)] += s
                    flag = g1 + '\t' + g2
                    flag1 = False
                    break
            if flag != g1 + '\t' + g2:
                flag1 = True
                g11, g22 = flag.split('\t')
                if len(eval(self.dataList[-1])) < self.MAX_DICT_LIMIT:
                    eval(self.dataList[-1])[(g11, g22)] = s_sum
                else:
                    setattr(self, 'DATA' + str(len(self.dataList)), {})
                    self.dataList.append('self.DATA' + str(len(self.dataList)))
                    eval(self.dataList[-1])[(g11, g22)] = s_sum
                    print('数据量过大，新增字典分片 ' + self.dataList[-1])
                    self.Log('数据量过大，新增字典分片 ' + self.dataList[-1])
                flag = g1 + '\t' + g2
                s_sum = 0.0
            s_sum += s
        if flag1:
            g11, g22 = flag.split('\t')
            falg_f = True
            for dict_i in self.dataList:
                dict_i = eval(dict_i)
                if (g11, g22) in dict_i.keys():
                    dict_i[(g11, g22)] += s_sum
                    falg_f = False
                    break
            if falg_f:
                eval(self.dataList[-1])[(g11, g22)] = s_sum
        return self.dataList

    def c1c2s(self, g2_group, number_num):
        self.Co1_Co2_score = []
        for x in range(0, len(g2_group) - 1):
            for y in range(x + 1, len(g2_group)):
                g1_a, score_a = g2_group[x]
                g1_b, score_b = g2_group[y]
                self.Co1_Co2_score.append((g1_a, g1_b, score_a * score_b))
                self.Co1_Co2_score.append((g1_b, g1_a, score_a * score_b))
                number_num += 2
        self.dataList = self.CoCos_dict(self.Co1_Co2_score)
        return number_num

    def MapReduce1(self):
        g1_flag = None
        g1_group = []
        g1_g2_s1 = []
        for Data, sheetName, number in self.readFile(self.READPATH, 1, '\t'):
            for line in Data:
                self.data.append([str(line[0]), str(line[1]), float(line[2])])
            break
        del Data
        self.data = sorted(self.data, key=operator.itemgetter(0))
        self.Log('Map1 ok !!!' + str(len(self.data)))
        print('Map1 ok !!!' + str(len(self.data)))
        while self.data:
            line = self.data.pop()
            if line[0] == '':
                continue
            g1, g2, s = line[0], line[1], line[2]
            if g1_flag is None:
                g1_flag = g1
            if g1_flag != g1:
                sum_1 = 0.0
                if len(g1_group) >= self.MAX_U:
                    self.Log(g1 + ' 预警域名共 ' + str(len(g1_group)) + ' 次， 超过 ' + str(self.MAX_U) + ' 次')
                    print(g1 + ' 预警域名共 ' + str(len(g1_group)) + ' 次， 超过 ' + str(self.MAX_U) + ' 次')
                    g1_group = [(g2, s)]
                    g1_flag = g1
                    continue
                for tuple in g1_group:
                    tuple_g2, tuple_s = tuple
                    sum_1 += pow(tuple_s, 2)
                sum_1 = math.sqrt(sum_1) + self.GYH
                for tuple in g1_group:
                    tuple_g2, tuple_s = tuple
                    if sum_1 == 0.0:
                        sum_1 = 9999999
                    g1_g2_s1.append([str(g1_flag), str(tuple_g2), tuple_s / sum_1])
                g1_flag = g1
                g1_group = []
            g1_group.append((g2, s))
        sum_1 = 0.0
        for tuple in g1_group:
            tuple_g2, tuple_s = tuple
            sum_1 += pow(tuple_s, 2)
        sum_1 = math.sqrt(sum_1) + self.GYH
        for tuple in g1_group:
            if sum_1 == 0.0:
                sum_1 = 9999999
            tuple_g2, tuple_s = tuple
            g1_g2_s1.append([str(g1_flag), str(tuple_g2), tuple_s / sum_1])
        self.g1_g2_s = sorted(g1_g2_s1, key=operator.itemgetter(1))
        del g1_g2_s1
        print('Reduce1 ok!!! ' + str(len(self.g1_g2_s)))
        self.Log('Reduce1 ok!!! ' + str(len(self.g1_g2_s)))

    def MapReduec2(self):
        e = 0
        number_num = 0
        g2_flag = None
        g2_group = []
        xx = 10
        len_ggs = len(self.g1_g2_s)
        for g1_g2_score in self.g1_g2_s:
            g1, g2, s = g1_g2_score
            e += 1
            if (int((e / len_ggs) * 100)) >= xx:
                print('Map2 ' + str(int((e / len_ggs) * 100)) + '%')
                self.Log('Map2 ' + str(int((e / len_ggs) * 100)) + '%')
                xx += 10
            if g2_flag is None:
                g2_flag = g2
            if g2_flag != g2:
                if len(g2_group) >= self.MAX_I:
                    print(g2_flag + '\t 被' + str(len(g2_group)) + ' 用户点击，超过' + str(self.MAX_I) + '最大限制，失去相关性计算要求')
                    self.Log(g2_flag + '\t 被' + str(len(g2_group)) + ' 用户点击，超过' + str(self.MAX_I) + '最大限制，失去相关性计算要求')
                    g2_flag = g2
                    g2_group = [(g1, s)]
                    continue
                number_num = self.c1c2s(g2_group, number_num)
                g2_flag = g2
                g2_group = []
            g2_group.append((g1, s))
        number_num = self.c1c2s(g2_group, number_num)
        self.Log('Map2 ok!!!')
        self.Log('Reduce2 ok!!!')
        self.Log('共现矩阵数: ' + str(number_num))
        print('Map2 ok!!!')
        print('Reduce2 ok!!!')
        print('共现矩阵数: ' + str(number_num))

    def MapReduce3(self):
        flag = None
        listnumber = self.LISTNUMBER
        for data_i in range(len(self.dataList)):
            self.Log('Map3_' + str(data_i))
            for g1_g2, s in dict(sorted(eval(self.dataList[data_i]).items(), key=lambda data: [data[0][0], data[1]],
                                        reverse=True)).items():
                g1, g2 = g1_g2
                if flag is None:
                    flag = g1
                if flag != g1:
                    flag = g1
                    listnumber = self.LISTNUMBER
                if flag == g1 and listnumber <= 0:
                    continue
                self.writeFile(self.WPATH, [[g1, g2, s]], splitf=',')
                listnumber -= 1
            print('Map3_' + str(data_i) + ' ok!!!')
            self.Log('Map3_' + str(data_i) + ' ok!!!')
        print('MapReduce3 ok !!!')
        print('ok!!!')
        self.Log('MapReduce3 ok !!!')
        self.Log('ok!!!')


if __name__ == '__main__':
    MapReduce()
